<!DOCTYPE html>
<html lang="zh-CN">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
  <meta name="theme-color" content="#222">
  <meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/safari-pinned-tab.svg" color="#222">
  <link rel="stylesheet" href="/css/main.css">
  <link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>
  <script id="hexo-configurations">
    var NexT = window.NexT ||
    {};
    var CONFIG = {
      "hostname": "cuiqingcai.com",
      "root": "/",
      "scheme": "Pisces",
      "version": "7.8.0",
      "exturl": false,
      "sidebar":
      {
        "position": "right",
        "width": 360,
        "display": "post",
        "padding": 18,
        "offset": 12,
        "onmobile": false,
        "widgets": [
          {
            "type": "image",
            "name": "阿布云",
            "enable": false,
            "url": "https://www.abuyun.com/http-proxy/introduce.html",
            "src": "https://qiniu.cuiqingcai.com/88au8.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "天验",
            "enable": true,
            "url": "https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850",
            "src": "https://qiniu.cuiqingcai.com/bco2a.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "华为云",
            "enable": false,
            "url": "https://activity.huaweicloud.com/2020_618_promotion/index.html?bpName=5f9f98a29e2c40b780c1793086f29fe2&bindType=1&salesID=wangyubei",
            "src": "https://qiniu.cuiqingcai.com/y42ik.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "张小鸡",
            "enable": false,
            "url": "http://www.zxiaoji.com/",
            "src": "https://qiniu.cuiqingcai.com/fm72f.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "Luminati",
            "src": "https://qiniu.cuiqingcai.com/ikkq9.jpg",
            "url": "https://luminati-china.io/?affiliate=ref_5fbbaaa9647883f5c6f77095",
            "width": "100%",
            "enable": false
      },
          {
            "type": "image",
            "name": "IPIDEA",
            "url": "http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc",
            "src": "https://qiniu.cuiqingcai.com/0ywun.png",
            "width": "100%",
            "enable": true
      },
          {
            "type": "tags",
            "name": "标签云",
            "enable": true
      },
          {
            "type": "categories",
            "name": "分类",
            "enable": true
      },
          {
            "type": "friends",
            "name": "友情链接",
            "enable": true
      },
          {
            "type": "hot",
            "name": "猜你喜欢",
            "enable": true
      }]
      },
      "copycode":
      {
        "enable": true,
        "show_result": true,
        "style": "mac"
      },
      "back2top":
      {
        "enable": true,
        "sidebar": false,
        "scrollpercent": true
      },
      "bookmark":
      {
        "enable": false,
        "color": "#222",
        "save": "auto"
      },
      "fancybox": false,
      "mediumzoom": false,
      "lazyload": false,
      "pangu": true,
      "comments":
      {
        "style": "tabs",
        "active": "gitalk",
        "storage": true,
        "lazyload": false,
        "nav": null,
        "activeClass": "gitalk"
      },
      "algolia":
      {
        "hits":
        {
          "per_page": 10
        },
        "labels":
        {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      },
      "localsearch":
      {
        "enable": true,
        "trigger": "auto",
        "top_n_per_article": 10,
        "unescape": false,
        "preload": false
      },
      "motion":
      {
        "enable": false,
        "async": false,
        "transition":
        {
          "post_block": "bounceDownIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      "path": "search.xml"
    };

  </script>
  <meta name="description" content="你好，我是悦创。 本篇将开启我自己啃代理池的心得，将逐步放送，因为代理池搭建较为复杂，这里我就尽可能把代理池分成几篇来讲，同时也保证，在我其他篇放出来之前，每一篇都是你们的新知识。 学习就像看小说一样，一次一篇就会显得额外的轻松！ 当你把学习当作某个娱乐的事情来做，你会发现不一样的世界！ 我们无法延长生命长度，但是我们延长生命宽度，学习编程就是扩展生命最有力的武器！ 1. 看完之后你会得到什么">
  <meta property="og:type" content="article">
  <meta property="og:title" content="多个站点同时抓取！怎么做到的？">
  <meta property="og:url" content="https://cuiqingcai.com/9447.html">
  <meta property="og:site_name" content="静觅">
  <meta property="og:description" content="你好，我是悦创。 本篇将开启我自己啃代理池的心得，将逐步放送，因为代理池搭建较为复杂，这里我就尽可能把代理池分成几篇来讲，同时也保证，在我其他篇放出来之前，每一篇都是你们的新知识。 学习就像看小说一样，一次一篇就会显得额外的轻松！ 当你把学习当作某个娱乐的事情来做，你会发现不一样的世界！ 我们无法延长生命长度，但是我们延长生命宽度，学习编程就是扩展生命最有力的武器！ 1. 看完之后你会得到什么">
  <meta property="og:locale" content="zh_CN">
  <meta property="og:image" content="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200602175929.png">
  <meta property="og:image" content="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200602213154.png">
  <meta property="og:image" content="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200616111544">
  <meta property="article:published_time" content="2020-08-02T09:24:34.000Z">
  <meta property="article:modified_time" content="2021-12-18T13:11:11.569Z">
  <meta property="article:author" content="崔庆才">
  <meta property="article:tag" content="崔庆才">
  <meta property="article:tag" content="静觅">
  <meta property="article:tag" content="PHP">
  <meta property="article:tag" content="Java">
  <meta property="article:tag" content="Python">
  <meta property="article:tag" content="Spider">
  <meta property="article:tag" content="爬虫">
  <meta property="article:tag" content="Web">
  <meta property="article:tag" content="Kubernetes">
  <meta property="article:tag" content="深度学习">
  <meta property="article:tag" content="机器学习">
  <meta property="article:tag" content="数据分析">
  <meta property="article:tag" content="网络">
  <meta property="article:tag" content="IT">
  <meta property="article:tag" content="技术">
  <meta property="article:tag" content="博客">
  <meta name="twitter:card" content="summary">
  <meta name="twitter:image" content="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200602175929.png">
  <link rel="canonical" href="https://cuiqingcai.com/9447.html">
  <script id="page-configurations">
    // https://hexo.io/docs/variables.html
    CONFIG.page = {
      sidebar: "",
      isHome: false,
      isPost: true,
      lang: 'zh-CN'
    };

  </script>
  <title>多个站点同时抓取！怎么做到的？ | 静觅</title>
  <meta name="google-site-verification" content="p_bIcnvirkFzG2dYKuNDivKD8-STet5W7D-01woA2fc" />
  <noscript>
    <style>
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-header
      {
        opacity: initial;
      }

      .use-motion .site-title,
      .use-motion .site-subtitle
      {
        opacity: initial;
        top: initial;
      }

      .use-motion .logo-line-before i
      {
        left: initial;
      }

      .use-motion .logo-line-after i
      {
        right: initial;
      }

    </style>
  </noscript>
  <link rel="alternate" href="/atom.xml" title="静觅" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-container">
          <div class="site-nav-toggle">
            <div class="toggle" aria-label="切换导航栏">
              <span class="toggle-line toggle-line-first"></span>
              <span class="toggle-line toggle-line-middle"></span>
              <span class="toggle-line toggle-line-last"></span>
            </div>
          </div>
          <div class="site-meta">
            <a href="/" class="brand" rel="start">
              <span class="logo-line-before"><i></i></span>
              <h1 class="site-title">静觅 <span class="site-subtitle"> 崔庆才的个人站点 </span>
              </h1>
              <span class="logo-line-after"><i></i></span>
            </a>
          </div>
          <div class="site-nav-right">
            <div class="toggle popup-trigger">
              <i class="fa fa-search fa-fw fa-lg"></i>
            </div>
          </div>
        </div>
        <nav class="site-nav">
          <ul id="menu" class="main-menu menu">
            <li class="menu-item menu-item-home">
              <a href="/" rel="section">首页</a>
            </li>
            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">文章列表</a>
            </li>
            <li class="menu-item menu-item-tags">
              <a href="/tags/" rel="section">文章标签</a>
            </li>
            <li class="menu-item menu-item-categories">
              <a href="/categories/" rel="section">文章分类</a>
            </li>
            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">关于博主</a>
            </li>
            <li class="menu-item menu-item-message">
              <a href="/message/" rel="section">给我留言</a>
            </li>
            <li class="menu-item menu-item-search">
              <a role="button" class="popup-trigger">搜索 </a>
            </li>
          </ul>
        </nav>
        <div class="search-pop-overlay">
          <div class="popup search-popup">
            <div class="search-header">
              <span class="search-icon">
                <i class="fa fa-search"></i>
              </span>
              <div class="search-input-container">
                <input autocomplete="off" autocapitalize="off" placeholder="搜索..." spellcheck="false" type="search" class="search-input">
              </div>
              <span class="popup-btn-close">
                <i class="fa fa-times-circle"></i>
              </span>
            </div>
            <div id="search-result">
              <div id="no-result">
                <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>
    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span>0%</span>
    </div>
    <div class="reading-progress-bar"></div>
    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content post posts-expand">
            <article itemscope itemtype="http://schema.org/Article" class="post-block single" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/9447.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h1 class="post-title" itemprop="name headline"> 多个站点同时抓取！怎么做到的？ </h1>
                <div class="post-meta">
                  <span class="post-meta-item">
                    <span class="post-meta-item-icon">
                      <i class="far fa-user"></i>
                    </span>
                    <span class="post-meta-item-text">作者</span>
                    <span><a href="/authors/黄家宝|AI悦创" class="author" itemprop="url" rel="index">黄家宝|AI悦创</a></span>
                  </span>
                  <span class="post-meta-item">
                    <span class="post-meta-item-icon">
                      <i class="far fa-calendar"></i>
                    </span>
                    <span class="post-meta-item-text">发表于</span>
                    <time title="创建时间：2020-08-02 17:24:34" itemprop="dateCreated datePublished" datetime="2020-08-02T17:24:34+08:00">2020-08-02</time>
                  </span>
                  <span class="post-meta-item">
                    <span class="post-meta-item-icon">
                      <i class="far fa-folder"></i>
                    </span>
                    <span class="post-meta-item-text">分类于</span>
                    <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                      <a href="/categories/Python/" itemprop="url" rel="index"><span itemprop="name">Python</span></a>
                    </span>
                  </span>
                  <span id="/9447.html" class="post-meta-item leancloud_visitors" data-flag-title="多个站点同时抓取！怎么做到的？" title="阅读次数">
                    <span class="post-meta-item-icon">
                      <i class="fa fa-eye"></i>
                    </span>
                    <span class="post-meta-item-text">阅读次数：</span>
                    <span class="leancloud-visitors-count"></span>
                  </span>
                  <span class="post-meta-item" title="本文字数">
                    <span class="post-meta-item-icon">
                      <i class="far fa-file-word"></i>
                    </span>
                    <span class="post-meta-item-text">本文字数：</span>
                    <span>17k</span>
                  </span>
                  <span class="post-meta-item" title="阅读时长">
                    <span class="post-meta-item-icon">
                      <i class="far fa-clock"></i>
                    </span>
                    <span class="post-meta-item-text">阅读时长 &asymp;</span>
                    <span>15 分钟</span>
                  </span>
                </div>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="advertisements">
                  <div class="item">
                    <a href="http://i0k.cn/4UUsd" target="_blank">
                      <img src="https://qiniu.cuiqingcai.com/dsdhf.jpg">
                    </a>
                  </div>
                </div>
                <p>你好，我是悦创。 本篇将开启我自己啃代理池的心得，将逐步放送，因为代理池搭建较为复杂，这里我就尽可能把代理池分成几篇来讲，同时也保证，在我其他篇放出来之前，每一篇都是你们的新知识。 学习就像看小说一样，一次一篇就会显得额外的轻松！ 当你把学习当作某个娱乐的事情来做，你会发现不一样的世界！ 我们无法延长生命长度，但是我们延长生命宽度，学习编程就是扩展生命最有力的武器！</p>
                <h2 id="1-看完之后你会得到什么"><a href="#1-看完之后你会得到什么" class="headerlink" title="1. 看完之后你会得到什么"></a>1. 看完之后你会得到什么</h2>
                <ul>
                  <li>返回 yield；</li>
                  <li>eval 的使用；</li>
                  <li>多个代理网站同时抓取；</li>
                  <li>使用异步测试代理是否可用；</li>
                  <li>Python 的元类编程简单介绍；</li>
                  <li>正则表达式、PyQuery 提取数据；</li>
                  <li>模块化编程；</li>
                </ul>
                <p>废话不多说，马上步入正题！</p>
                <h2 id="2-你需要的准备"><a href="#2-你需要的准备" class="headerlink" title="2. 你需要的准备"></a>2. 你需要的准备</h2>
                <p>在学习本篇文章时，希望你已经具备如下技能或者知识点：</p>
                <ol>
                  <li>Python 环境（推荐 Python 3.7+）；</li>
                  <li>Python 爬虫常用库；</li>
                  <li>Python 基本语法；</li>
                  <li>面向对象编程；</li>
                  <li>yield、eval 的使用；</li>
                  <li>模块化编程；</li>
                </ol>
                <h2 id="3-课前预习知识点"><a href="#3-课前预习知识点" class="headerlink" title="3. 课前预习知识点"></a>3. 课前预习知识点</h2>
                <p>对于代理池的搭建呢，虽然我已经尽可能地照顾到绝大多数地小白，把代理地的搭建呢，进行了拆分，不过对于培训机构或者自学不是特别精进的小伙伴来说还是有些难度的，对于这些呢？我这里也给大家准备了知识点的扫盲。以下内容节选自我个人博客文章，这里为了扫盲就提供必要部分，想看完整的文章可以点击此链接：<a href="https://www.aiyc.top/archives/605.html" target="_blank" rel="noopener">https://www.aiyc.top/archives/605.html</a></p>
                <h3 id="3-1-Python-的元类编程"><a href="#3-1-Python-的元类编程" class="headerlink" title="3.1 Python 的元类编程"></a>3.1 Python 的元类编程</h3>
                <p>你好，我是悦创。 好久不见，最近在啃数学、Java、英语没有来更新公众号，那么今天就来更新一下啦！ 又到了每日一啃，啃代码的小悦。今天我遇到了 Python 原类，然而我啥也不懂。只能靠百度和谷歌了，主要还是用谷歌来查啦，百度前几条永远是广告准确度也不行（个人观点）,也顺便参考了几个博客：廖雪峰网站，添加了一点自己的观点和理解。</p>
                <h4 id="3-1-1-type"><a href="#3-1-1-type" class="headerlink" title="3.1.1 type()"></a>3.1.1 type()</h4>
                <p>动态语言和静态语言最大的不同，就是函数和类的定义，不是编译时定义的，而是运行时动态创建的。 比方说我们要定义一个 <code>Hello</code> 的 class，就写一个 <code>hello.py</code> 模块（这里我实际创建的是：<code>the_test_code_project.py</code> 模块）：</p>
                <figure class="highlight python">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Hello</span><span class="params">(object)</span>:</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">hello</span><span class="params">(self, name=<span class="string">'world'</span>)</span>:</span></span><br><span class="line">print(<span class="string">'Hello, %s.'</span> % name)</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>当 Python 解释器载入 <code>hello</code> 模块时，就会依次执行该模块的所有语句，执行结果就是动态创建出一个 <code>Hello</code> 的 class 对象，测试如下：</p>
                <figure class="highlight isbl">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="variable"><span class="keyword">if</span></span> <span class="variable">__name__</span> == <span class="string">'__main__'</span>:</span><br><span class="line"><span class="variable">h</span> = <span class="function"><span class="title">Hello</span>()</span></span><br><span class="line"><span class="variable">h.hello</span>()</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable">Hello</span>))</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable">h</span>))</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>运行结果如下：</p>
                <figure class="highlight ceylon">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">Hello, world.</span><br><span class="line">&lt;<span class="keyword">class</span> <span class="string">'type'</span>&gt;</span><br><span class="line">&lt;<span class="keyword">class</span> <span class="string">'__main__.Hello'</span>&gt;</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>其中，上面的输出结果：<code>__main__.Hello</code> 等价于 <code>&lt;class &#39;the_test_code_project.Hello&#39;&gt;</code> 运行的方式不同显示的方式也不同，但含义是一样的。 <img src="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200602175929.png" alt=""> <code>type()</code> 函数可以查看一个类型或变量的类型，为了让小白更轻松，我也写了个例子：</p>
                <figure class="highlight vim">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="keyword">number</span> = <span class="number">12</span></span><br><span class="line"><span class="built_in">string</span> = <span class="string">'Hello AIYC!'</span></span><br><span class="line">float_number = <span class="number">12.1</span></span><br><span class="line">list_data = [<span class="number">1</span>, <span class="string">'2'</span>, <span class="number">3.5</span>, <span class="string">'AIYC'</span>] # 可变</span><br><span class="line">tuples = (<span class="number">1</span>, <span class="string">'2'</span>, <span class="number">3.5</span>, <span class="string">'AIYC'</span>) # 不可变</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line"><span class="keyword">print</span>(<span class="built_in">type</span>(<span class="keyword">number</span>))</span><br><span class="line"><span class="keyword">print</span>(<span class="built_in">type</span>(<span class="built_in">string</span>))</span><br><span class="line"><span class="keyword">print</span>(<span class="built_in">type</span>(float_number))</span><br><span class="line"><span class="keyword">print</span>(<span class="built_in">type</span>(list_data))</span><br><span class="line"><span class="keyword">print</span>(<span class="built_in">type</span>(tuples))</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>运行结果：</p>
                <figure class="highlight angelscript">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">&lt;<span class="keyword">class</span> '<span class="symbol">int</span>'&gt;</span><br><span class="line">&lt;<span class="symbol">class</span> '<span class="symbol">str</span>'&gt;</span><br><span class="line">&lt;<span class="symbol">class</span> '<span class="symbol">float</span>'&gt;</span><br><span class="line">&lt;<span class="symbol">class</span> '<span class="symbol">list</span>'&gt;</span><br><span class="line">&lt;<span class="symbol">class</span> '<span class="symbol">tuple</span>'&gt;</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p><code>Hello</code> 是一个 class，它的类型就是 <code>type</code> ，而 <code>h</code> 是一个实例，它的类型就是class <code>Hello</code>。 我们说 class 的定义是运行时动态创建的，而创建 class 的方法就是使用 <code>type()</code> 函数。 <code>type()</code> 函数既可以返回一个对象的类型，又可以创建出新的类型，比如，我们可以通过 <code>type()</code> 函数创建出<code>Hello</code> 类，而无需通过 <code>class Hello(object)...</code> 的定义：</p>
                <figure class="highlight delphi">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">def fn(self, <span class="keyword">name</span>=<span class="string">'world'</span>): # 先定义函数</span><br><span class="line">print(<span class="string">'Hello, %s.'</span> % <span class="keyword">name</span>)</span><br><span class="line"></span><br><span class="line">Hello = <span class="keyword">type</span>(<span class="string">'Hello'</span>, (<span class="keyword">object</span>,), dict(hello=fn)) # 创建 Hello <span class="keyword">class</span></span><br><span class="line"># Hello = <span class="keyword">type</span>(<span class="string">'Class_Name'</span>, (<span class="keyword">object</span>,), dict(hello=fn)) # 创建 Hello <span class="keyword">class</span></span><br><span class="line"># <span class="keyword">type</span>(类名, 父类的元组（针对继承的情况，可以为空），包含属性的字典（名称和值）)</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>我们接下来来调用一下代码，看输出的结果如何：</p>
                <figure class="highlight isbl">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="variable"><span class="keyword">if</span></span> <span class="variable">__name__</span> == <span class="string">'__main__'</span>:</span><br><span class="line"><span class="variable">h</span> = <span class="function"><span class="title">Hello</span>()</span></span><br><span class="line"><span class="variable">h.hello</span>()</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable">Hello</span>))</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable">h</span>))</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>这里推荐写成：<code>if __name__ == &#39;__main__&#39;:</code> 使代码更加的规范。 运行结果：</p>
                <figure class="highlight ceylon">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">Hello, world.</span><br><span class="line">&lt;<span class="keyword">class</span> <span class="string">'type'</span>&gt;</span><br><span class="line">&lt;<span class="keyword">class</span> <span class="string">'__main__.Hello'</span>&gt;</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>要创建一个 class 对象，<code>type()</code> 函数依次传入 3 个参数：</p>
                <ol>
                  <li>class 的名称；</li>
                  <li>继承的父类集合，注意 Python 支持多重继承，如果只有一个父类，别忘了 tuple 的单元素写法；（这个个 tuple 单元素写法起初本人不太理解，然后一查并认真观察了一下上面的代码就想起来 tuple 单元素写法需要加逗号（，），就是你必须这么写：<code>tuple_1 = (1,)</code> 而不能这么写：<code>tuple_2 = (1)</code> ，<code>tuple_2 = (1)</code> 的写法，Python 会自动认为是一个整数而不是一个元组）</li>
                  <li>class 的方法名称与函数绑定，这里我们把函数 <code>fn</code> 绑定到方法名 <code>hello</code> 上。</li>
                </ol>
                <p>通过 <code>type()</code> 函数创建的类和直接写 class 是完全一样的，因为 Python 解释器遇到 class 定义时，仅仅是扫描一下 class 定义的语法，然后调用 <code>type()</code> 函数创建出 class。（直接 Class 创建也是） 正常情况下，我们都用 <code>class Xxx...</code> 来定义类，但是，<code>type()</code> 函数也允许我们动态创建出类来，也就是说，动态语言本身支持运行期动态创建类，这和静态语言有非常大的不同，要在静态语言运行期创建类，必须构造源代码字符串再调用编译器，或者借助一些工具生成字节码实现，本质上都是动态编译，会非常复杂。</p>
                <h4 id="3-1-2-metaclass"><a href="#3-1-2-metaclass" class="headerlink" title="3.1.2 metaclass"></a>3.1.2 metaclass</h4>
                <p>除了使用 <code>type()</code> 动态创建类以外，要控制类的 <strong>创建行为</strong> ，还可以使用 metaclass。 metaclass，直译为元类，简单的解释就是：</p>
                <ul>
                  <li>当我们定义了类以后，就可以根据这个类创建出实例，所以：先定义类，然后创建实例。</li>
                </ul>
                <p><img src="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200602213154.png" alt="image-20200602213149088"></p>
                <ul>
                  <li>但是如果我们想创建出类呢？</li>
                </ul>
                <p>那就必须根据 metaclass 创建出类，所以：先定义 metaclass ，然后创建类。连接起来就是：先定义 metaclass ，就可以创建类，最后创建实例。 所以，<strong>metaclass 允许你创建类或者修改类</strong> 。换句话说，你可以把类看成是 metaclass 创建出来的“实例”。</p>
                <h6 id="metaclass-是-Python-面向对象里最难理解，也是最难使用的魔术代码。正常情况下，你不会碰到需要使用metaclass的情况，所以，以下内容看不懂也没关系，因为基本上你不会用到。（然而还是被我遇见了，而且还是看不懂，但经过大佬的指点就只是知道如何使用，但并不了解其中的原理，所以才有了此篇。"><a href="#metaclass-是-Python-面向对象里最难理解，也是最难使用的魔术代码。正常情况下，你不会碰到需要使用metaclass的情况，所以，以下内容看不懂也没关系，因为基本上你不会用到。（然而还是被我遇见了，而且还是看不懂，但经过大佬的指点就只是知道如何使用，但并不了解其中的原理，所以才有了此篇。" class="headerlink" title="metaclass 是 Python 面向对象里最难理解，也是最难使用的魔术代码。正常情况下，你不会碰到需要使用metaclass的情况，所以，以下内容看不懂也没关系，因为基本上你不会用到。（然而还是被我遇见了，而且还是看不懂，但经过大佬的指点就只是知道如何使用，但并不了解其中的原理，所以才有了此篇。)"></a>metaclass 是 Python 面向对象里最难理解，也是最难使用的魔术代码。正常情况下，你不会碰到需要使用metaclass的情况，所以，以下内容看不懂也没关系，因为基本上你不会用到。（然而还是被我遇见了，而且还是看不懂，但经过大佬的指点就只是知道如何使用，但并不了解其中的原理，所以才有了此篇。)</h6>
                <p>我们先看一个简单的例子，这个 metaclass 可以给我们自定义的 MyList 增加一个 <code>add</code> 方法：</p>
                <figure class="highlight reasonml">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="keyword">class</span> <span class="constructor">ListMetaclass(<span class="params">type</span>)</span>:</span><br><span class="line">def <span class="constructor">__new__(<span class="params">cls</span>, <span class="params">name</span>, <span class="params">bases</span>, <span class="params">attrs</span>)</span>:</span><br><span class="line">attrs<span class="literal">['<span class="identifier">add</span>']</span> = lambda self, value: self.append(value) # 加上新的方法</span><br><span class="line">return <span class="keyword">type</span>.<span class="constructor">__new__(<span class="params">cls</span>, <span class="params">name</span>, <span class="params">bases</span>, <span class="params">attrs</span>)</span> # 返回修改后的定义</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <blockquote>
                  <p>定义 <code>ListMetaclass</code> ，按照默认习惯，metaclass 的类名总是以 Metaclass 结尾，以便清楚地表示这是一个metaclass 。</p>
                </blockquote>
                <p>有了 ListMetaclass ，我们在定义类的时候还要指示使用 ListMetaclass 来定制类，传入关键字参数 <code>metaclass</code> ：</p>
                <figure class="highlight angelscript">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="keyword">class</span> <span class="symbol">MyList</span>(<span class="symbol">list, <span class="symbol">metaclass</span></span>=<span class="symbol">ListMetaclass</span>):</span><br><span class="line"><span class="symbol">pass</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>当我们传入关键字参数 <code>metaclass</code> 时，魔术就生效了，它指示 Python 解释器在创建 <code>MyList</code> 时，要通过<code>ListMetaclass.__new__()</code> 来创建，在此，我们可以修改类的定义，比如，加上新的方法，然后，返回修改后的定义。 <code>__new__()</code> 方法接收到的参数依次是：</p>
                <ol>
                  <li>当前准备创建的类的对象；</li>
                  <li>类的名字；</li>
                  <li>类继承的父类集合；</li>
                  <li>类的方法集合。</li>
                </ol>
                <p>测试一下 <code>MyList</code> 是否可以调用 <code>add()</code> 方法：</p>
                <figure class="highlight routeros">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">L = MyList()</span><br><span class="line">L.<span class="builtin-name">add</span>(1)</span><br><span class="line"><span class="builtin-name">print</span>(L)</span><br><span class="line"></span><br><span class="line"><span class="comment"># 输出</span></span><br><span class="line">[1]</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>而普通的 <code>list</code> 没有 <code>add()</code> 方法：</p>
                <figure class="highlight vim">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">\&gt;&gt;&gt; L2 = <span class="keyword">list</span>()</span><br><span class="line">&gt;&gt;&gt; L2.<span class="built_in">add</span>(<span class="number">1</span>)</span><br><span class="line">Traceback (most recent <span class="keyword">call</span> <span class="keyword">last</span>):</span><br><span class="line">File <span class="string">"&lt;stdin&gt;"</span>, <span class="built_in">line</span> <span class="number">1</span>, in <span class="symbol">&lt;module&gt;</span></span><br><span class="line">AttributeError: <span class="string">'list'</span> object <span class="built_in">has</span> <span class="keyword">no</span> attribute <span class="string">'add'</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p><strong>这时候，我想你应该和我会有同样的问题，动态修改有什么意义？</strong> <strong>直接在 <code>MyList</code> 定义中写上 <code>add()</code> 方法不是更简单吗？</strong> 正常情况下，确实应该直接写，我觉得通过 metaclass 修改纯属变态。</p>
                <h3 id="3-2-Python-eval-函数"><a href="#3-2-Python-eval-函数" class="headerlink" title="3.2 Python eval() 函数"></a>3.2 Python eval() 函数</h3>
                <p>对于 Python 的 eval 呢，大部分人是这么定义的：eval() 函数用来执行一个字符串表达式，并返回表达式的值。 这句话，有可能对于新手来说并不是非常好理解，我们还是用例子来顿悟一下吧。 以下是 eval() 方法的语法：</p>
                <figure class="highlight stylus">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="function"><span class="title">eval</span><span class="params">(expression[, globals[, locals]])</span></span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p><strong>参数</strong></p>
                <ul>
                  <li>expression — 表达式。</li>
                  <li>globals — 变量作用域，全局命名空间，如果被提供，则必须是一个字典对象。</li>
                  <li>locals — 变量作用域，局部命名空间，如果被提供，可以是任何映射对象。</li>
                </ul>
                <p><strong>返回值</strong> 返回表达式计算结果。 <strong>实际操作</strong></p>
                <figure class="highlight angelscript">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">In [<span class="number">3</span>]: x = <span class="number">7</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">4</span>]: eval( <span class="string">'3 * x'</span> )</span><br><span class="line">Out[<span class="number">4</span>]: <span class="number">21</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">5</span>]: eval(<span class="string">'pow(2,2)'</span>)</span><br><span class="line">Out[<span class="number">5</span>]: <span class="number">4</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">6</span>]: eval(<span class="string">'2 + 2'</span>)</span><br><span class="line">Out[<span class="number">6</span>]: <span class="number">4</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">7</span>]: n=<span class="number">81</span></span><br><span class="line"></span><br><span class="line">In [<span class="number">8</span>]: eval(<span class="string">"n + 4"</span>)</span><br><span class="line">Out[<span class="number">8</span>]: <span class="number">85</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>再来个函数的操作：</p>
                <figure class="highlight armasm">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="symbol">In</span> [<span class="number">1</span>]: <span class="keyword">str1 </span>= <span class="string">"print('Hello World')"</span></span><br><span class="line"></span><br><span class="line"><span class="symbol">In</span> [<span class="number">2</span>]: eval(<span class="keyword">str1)</span></span><br><span class="line"><span class="keyword">Hello </span>World</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>ok，到此零基础小白关怀文章就完成了，我就不继续赘述啦！ 看到这里，你们的身体还行吗？ <img src="https://images-aiyc-1301641396.cos.ap-guangzhou.myqcloud.com/20200616111544" alt="真叫人头大"> 我正在的干货要开始了！ 进入我们的 show time 环节，起飞了、起飞了，做好准备了！</p>
                <h2 id="4-抓取免费代理"><a href="#4-抓取免费代理" class="headerlink" title="4. 抓取免费代理"></a>4. 抓取免费代理</h2>
                <p>这里呢，我就不再带带大家手摸手的教学如何抓取代理了，因为当你能看这篇文章时，相信你已经对爬虫是有所入门了，如果还没入门的小伙伴可以关注本公众号，往期文章也有零基础教学和基础的课程资源，可以公众号后台回复：Python爬虫。即可获取资源，如果失效也别担心加小悦好友即可！（资源多多噢！）</p>
                <blockquote>
                  <p>小白不建议报名培训机构，毕竟现在培训结构收智商税比较多，还是需要多多鉴别噢！</p>
                </blockquote>
                <h3 id="4-1-目标的代理网站"><a href="#4-1-目标的代理网站" class="headerlink" title="4.1 目标的代理网站"></a>4.1 目标的代理网站</h3>
                <ol>
                  <li>快代理：<a href="https://www.kuaidaili.com/" target="_blank" rel="noopener">https://www.kuaidaili.com/</a></li>
                  <li>西刺代理：<a href="https://www.xicidaili.com" target="_blank" rel="noopener">https://www.xicidaili.com</a></li>
                  <li>66代理：<a href="http://www.66ip.cn/" target="_blank" rel="noopener">http://www.66ip.cn/</a></li>
                  <li>无忧代理：<a href="http://www.data5u.com" target="_blank" rel="noopener">http://www.data5u.com</a></li>
                  <li>开心代理-高匿：<a href="http://www.kxdaili.com/dailiip/" target="_blank" rel="noopener">http://www.kxdaili.com/dailiip/</a></li>
                  <li>云代理：<a href="http://www.ip3366.net/free/" target="_blank" rel="noopener">http://www.ip3366.net/free/</a></li>
                </ol>
                <p>对于每个网站的代码呢，具体实现也是比较简单的，这里我就不做过多的赘述啦！ 接下来我都直接上代码，不过在上代理的爬虫代码前，我们需要来写个元类，代码如下：</p>
                <figure class="highlight python">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ProxyMetaclass</span><span class="params">(type)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">定义 ProxyMetaclass ，按照默认习惯，metaclass 的类名总是以 Metaclass 结尾，以便清楚地表示这是一个metaclass ：</span></span><br><span class="line"><span class="string">元类，在 FreeProxyGetter 类中加入</span></span><br><span class="line"><span class="string">CrawlFunc_list 和 CrawlFuncCount</span></span><br><span class="line"><span class="string">两个参数，分别表示爬虫函数，和爬虫函数的数量。</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__new__</span><span class="params">(cls, name, bases, attrs)</span>:</span></span><br><span class="line">count = <span class="number">0</span></span><br><span class="line">attrs[<span class="string">'CrawlFunc_List'</span>] = [] <span class="comment"># 添加：CrawlFunc_List 列表方法</span></span><br><span class="line"><span class="keyword">for</span> k, v <span class="keyword">in</span> attrs.items():</span><br><span class="line"><span class="keyword">if</span> <span class="string">'crawl_'</span> <span class="keyword">in</span> k:</span><br><span class="line"><span class="comment"># 判断这个函数里面是否携带 crawl_ 也就是利用 value in xxx</span></span><br><span class="line">attrs[<span class="string">'CrawlFunc_List'</span>].append(k)</span><br><span class="line">count += <span class="number">1</span></span><br><span class="line">attrs[<span class="string">'CrawlFuncCount'</span>] = count <span class="comment"># 检测添加的函数的总数量</span></span><br><span class="line"><span class="keyword">return</span> type.__new__(cls, name, bases, attrs) <span class="comment"># 返回所修改的</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>详细的代码含义，已经写在上面了，具体的这里 就不赘述了，如果泥有任何不理解的可以去点击阅读原文在我的博客网站下留言，和后台回复数字“3”，加小编好友，拉你入交流群。</p>
                <h3 id="4-2-代码编写"><a href="#4-2-代码编写" class="headerlink" title="4.2 代码编写"></a>4.2 代码编写</h3>
                <h4 id="4-2-1-请求函数编写"><a href="#4-2-1-请求函数编写" class="headerlink" title="4.2.1 请求函数编写"></a>4.2.1 请求函数编写</h4>
                <p>单独保存成一个文件：<strong>utils.py</strong></p>
                <figure class="highlight python">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">project = 'Code', file_name = 'utils.py', author = 'AI悦创'</span></span><br><span class="line"><span class="string">time = '2020/6/1 12:34', product_name = PyCharm, 公众号：AI悦创</span></span><br><span class="line"><span class="string">code is far away from bugs with the god animal protecting</span></span><br><span class="line"><span class="string">I love animals. They taste delicious.</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> aiohttp</span><br><span class="line"><span class="keyword">from</span> requests.exceptions <span class="keyword">import</span> ConnectionError</span><br><span class="line"><span class="keyword">from</span> fake_useragent <span class="keyword">import</span> UserAgent,FakeUserAgentError</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_page</span><span class="params">(url, options=&#123;&#125;)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">构造随机请求头，如果不理解的可以阅读此文章：</span></span><br><span class="line"><span class="string">两行代码设置 Scrapy UserAgent：https://www.aiyc.top/archives/533.html</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">ua = UserAgent()</span><br><span class="line"><span class="keyword">except</span> FakeUserAgentError:</span><br><span class="line"><span class="keyword">pass</span></span><br><span class="line"><span class="comment"># 生成随机的请求头，加 try...except... 使代码更加健壮</span></span><br><span class="line">base_headers = &#123;</span><br><span class="line"><span class="string">'User-Agent'</span>: ua.random,</span><br><span class="line"><span class="string">'Accept-Encoding'</span>: <span class="string">'gzip, deflate, sdch'</span>,</span><br><span class="line"><span class="string">'Accept-Language'</span>: <span class="string">'zh-CN,zh;q=0.8'</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="comment"># 如果使用者有传入请求头，则将此请求头和随机生成的合成在一起</span></span><br><span class="line">headers = dict(base_headers, **options)</span><br><span class="line"><span class="comment"># 当前请求的 url</span></span><br><span class="line">print(<span class="string">'Getting'</span>, url)</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">r = requests.get(url, headers=headers)</span><br><span class="line">print(<span class="string">'Getting result'</span>, url, r.status_code)</span><br><span class="line"><span class="keyword">if</span> r.status_code == <span class="number">200</span>:</span><br><span class="line"><span class="keyword">return</span> r.text</span><br><span class="line"><span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"><span class="keyword">except</span> ConnectionError:</span><br><span class="line">print(<span class="string">'Crawling Failed'</span>, url)</span><br><span class="line"><span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Downloader</span><span class="params">(object)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">一个异步下载器，可以对代理源异步抓取，但是容易被 BAN。</span></span><br><span class="line"><span class="string">self._htmls: 把请求的 html 放入列表当中</span></span><br><span class="line"><span class="string">升级版的请求类</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, urls)</span>:</span></span><br><span class="line">self.urls = urls</span><br><span class="line">self._htmls = []</span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">def</span> <span class="title">download_single_page</span><span class="params">(self, url)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">下载单页</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">async</span> <span class="keyword">with</span> aiohttp.ClientSession() <span class="keyword">as</span> session:</span><br><span class="line"><span class="keyword">async</span> <span class="keyword">with</span> session.get(url) <span class="keyword">as</span> response:</span><br><span class="line">self._htmls.append(<span class="keyword">await</span> response.text())</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">download</span><span class="params">(self)</span>:</span></span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">tasks = [self.download_single_page(url) <span class="keyword">for</span> url <span class="keyword">in</span> self.urls]</span><br><span class="line">loop.run_until_complete(asyncio.wait(tasks))</span><br><span class="line"></span><br><span class="line"><span class="meta">@property</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">htmls</span><span class="params">(self)</span>:</span></span><br><span class="line">self.download()</span><br><span class="line"><span class="keyword">return</span> self._htmls</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>上面请求函数编写完成之后，我们就需要在抓取代理的代码文件中，进行导包，代码如下：</p>
                <figure class="highlight capnproto">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="comment"># files:Spider.py</span></span><br><span class="line"><span class="keyword">from</span> utils <span class="keyword">import</span> get_page</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <h4 id="4-2-2-代理抓取代码"><a href="#4-2-2-代理抓取代码" class="headerlink" title="4.2.2 代理抓取代码"></a>4.2.2 代理抓取代码</h4>
                <p>导入所需要的库：</p>
                <figure class="highlight elm">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="title">from</span> utils <span class="keyword">import</span> get_page</span><br><span class="line"><span class="title">from</span> pyquery <span class="keyword">import</span> PyQuery <span class="keyword">as</span> pq</span><br><span class="line"><span class="keyword">import</span> re</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>接下来我们需要创建一个类 FreeProxyGetter 并使用元类创建。</p>
                <figure class="highlight angelscript">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="keyword">class</span> <span class="symbol">FreeProxyGetter</span>(<span class="symbol">object, <span class="symbol">metaclass</span></span>=<span class="symbol">ProxyMetaclass</span>):</span><br><span class="line"><span class="symbol">pass</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <h5 id="1-快代理"><a href="#1-快代理" class="headerlink" title="1. 快代理"></a>1. 快代理</h5>
                <figure class="highlight python">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">crawl_kuaidaili</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">快代理</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">for</span> page <span class="keyword">in</span> range(<span class="number">1</span>, <span class="number">4</span>):</span><br><span class="line"><span class="comment"># 国内高匿代理</span></span><br><span class="line">start_url = <span class="string">'https://www.kuaidaili.com/free/inha/&#123;&#125;/'</span>.format(page)</span><br><span class="line">html = get_page(start_url)</span><br><span class="line"><span class="comment"># print(html)</span></span><br><span class="line">pattern = re.compile(</span><br><span class="line"><span class="string">'&lt;td data-title="IP"&gt;(.*)&lt;/td&gt;s*&lt;td data-title="PORT"&gt;(w+)&lt;/td&gt;'</span></span><br><span class="line">)</span><br><span class="line"><span class="comment"># s * 匹配空格，起到换行作用</span></span><br><span class="line"><span class="comment"># ip_addres = re.findall(pattern, html) # 写法一</span></span><br><span class="line">ip_addres = pattern.findall(str(html))</span><br><span class="line"><span class="keyword">for</span> adress, port <span class="keyword">in</span> ip_addres:</span><br><span class="line"><span class="comment"># print(adress, port)</span></span><br><span class="line">result = <span class="string">f"<span class="subst">&#123;adress&#125;</span>:<span class="subst">&#123;port&#125;</span>"</span>.strip()</span><br><span class="line"><span class="keyword">yield</span> result</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <h5 id="2-西刺代理"><a href="#2-西刺代理" class="headerlink" title="2. 西刺代理"></a>2. 西刺代理</h5>
                <figure class="highlight python">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">crawl_xicidaili</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">西刺代理</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">for</span> page <span class="keyword">in</span> range(<span class="number">1</span>, <span class="number">4</span>):</span><br><span class="line">start_url = <span class="string">'https://www.xicidaili.com/wt/&#123;&#125;'</span>.format(page)</span><br><span class="line">html = get_page(start_url)</span><br><span class="line"><span class="comment"># print(html)</span></span><br><span class="line">ip_adress = re.compile(</span><br><span class="line"><span class="string">'&lt;td class="country"&gt;&lt;img src="//fs.xicidaili.com/images/flag/cn.png" alt="Cn" /&gt;&lt;/td&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;'</span></span><br><span class="line">)</span><br><span class="line"><span class="comment"># s* 匹配空格，起到换行作用</span></span><br><span class="line">re_ip_adress = ip_adress.findall(str(html))</span><br><span class="line"><span class="keyword">for</span> adress, port <span class="keyword">in</span> re_ip_adress:</span><br><span class="line">result = <span class="string">f"<span class="subst">&#123;adress&#125;</span>:<span class="subst">&#123;port&#125;</span>"</span>.strip()</span><br><span class="line"><span class="keyword">yield</span> result</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <h5 id="3-66代理"><a href="#3-66代理" class="headerlink" title="3. 66代理"></a>3. 66代理</h5>
                <figure class="highlight routeros">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">def crawl_daili66(self, <span class="attribute">page_count</span>=4):</span><br><span class="line"><span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">66代理</span></span><br><span class="line"><span class="string">:param page_count:</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"</span><span class="string">""</span></span><br><span class="line">start_url = <span class="string">'http://www.66ip.cn/&#123;&#125;.html'</span></span><br><span class="line">urls = [start_url.format(page) <span class="keyword">for</span><span class="built_in"> page </span><span class="keyword">in</span> range(1, page_count + 1)]</span><br><span class="line"><span class="keyword">for</span> url <span class="keyword">in</span> urls:</span><br><span class="line"><span class="builtin-name">print</span>(<span class="string">'Crawling'</span>, url)</span><br><span class="line">html = get_page(url)</span><br><span class="line"><span class="keyword">if</span> html:</span><br><span class="line">doc = pq(html)</span><br><span class="line">trs = doc(<span class="string">'.containerbox table tr:gt(0)'</span>).items()</span><br><span class="line"><span class="keyword">for</span> tr <span class="keyword">in</span> trs:</span><br><span class="line">ip = tr.<span class="builtin-name">find</span>(<span class="string">'td:nth-child(1)'</span>).text()</span><br><span class="line">port = tr.<span class="builtin-name">find</span>(<span class="string">'td:nth-child(2)'</span>).text()</span><br><span class="line">yield <span class="string">':'</span>.join([ip, port])</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <h5 id="4-无忧代理"><a href="#4-无忧代理" class="headerlink" title="4. 无忧代理"></a>4. 无忧代理</h5>
                <figure class="highlight python">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">crawl_data5u</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">无忧代理</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line">start_url = <span class="string">'http://www.data5u.com'</span></span><br><span class="line">html = get_page(start_url)</span><br><span class="line"><span class="comment"># print(html)</span></span><br><span class="line">ip_adress = re.compile(</span><br><span class="line"><span class="string">'&lt;ul class="l2"&gt;s*&lt;span&gt;&lt;li&gt;(.*?)&lt;/li&gt;&lt;/span&gt;s*&lt;span style="width: 100px;"&gt;&lt;li class=".*"&gt;(.*?)&lt;/li&gt;&lt;/span&gt;'</span></span><br><span class="line">)</span><br><span class="line"><span class="comment"># s * 匹配空格，起到换行作用</span></span><br><span class="line">re_ip_adress = ip_adress.findall(str(html))</span><br><span class="line"><span class="keyword">for</span> adress, port <span class="keyword">in</span> re_ip_adress:</span><br><span class="line">result = <span class="string">f"<span class="subst">&#123;adress&#125;</span>:<span class="subst">&#123;port&#125;</span>"</span></span><br><span class="line"><span class="keyword">yield</span> result.strip()</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <h5 id="5-开心代理-高匿"><a href="#5-开心代理-高匿" class="headerlink" title="5. 开心代理-高匿"></a>5. 开心代理-高匿</h5>
                <figure class="highlight python">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">crawl_kxdaili</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">开心代理-高匿</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">1</span>, <span class="number">4</span>):</span><br><span class="line">start_url = <span class="string">'http://www.kxdaili.com/dailiip/1/&#123;&#125;.html'</span>.format(i)</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">html = requests.get(start_url)</span><br><span class="line"><span class="keyword">if</span> html.status_code == <span class="number">200</span>:</span><br><span class="line">html.encoding = <span class="string">'utf-8'</span></span><br><span class="line"><span class="comment"># print(html.text)</span></span><br><span class="line">ip_adress = re.compile(<span class="string">'&lt;tr.*?&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;'</span>)</span><br><span class="line"><span class="comment"># s* 匹配空格，起到换行作用</span></span><br><span class="line">re_ip_adress = ip_adress.findall(str(html.text))</span><br><span class="line"><span class="keyword">for</span> adress, port <span class="keyword">in</span> re_ip_adress:</span><br><span class="line">result = <span class="string">f"<span class="subst">&#123;adress&#125;</span>:<span class="subst">&#123;port&#125;</span>"</span></span><br><span class="line"><span class="keyword">yield</span> result.strip()</span><br><span class="line"><span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"><span class="keyword">except</span>:</span><br><span class="line"><span class="keyword">pass</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <h5 id="6-云代理"><a href="#6-云代理" class="headerlink" title="6. 云代理"></a>6. 云代理</h5>
                <figure class="highlight python">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">IP3366Crawler</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">云代理</span></span><br><span class="line"><span class="string">parse html file to get proxies</span></span><br><span class="line"><span class="string">:return:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line">start_url = <span class="string">'http://www.ip3366.net/free/?stype=1&amp;page=&#123;page&#125;'</span></span><br><span class="line">urls = [start_url.format(page=i) <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">1</span>, <span class="number">8</span>)]</span><br><span class="line"><span class="comment"># s * 匹配空格，起到换行作用</span></span><br><span class="line">ip_address = re.compile(<span class="string">'&lt;tr&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;s*&lt;td&gt;(.*?)&lt;/td&gt;'</span>)</span><br><span class="line"><span class="keyword">for</span> url <span class="keyword">in</span> urls:</span><br><span class="line">html = get_page(url)</span><br><span class="line">re_ip_address = ip_address.findall(str(html))</span><br><span class="line"><span class="keyword">for</span> adress, port <span class="keyword">in</span> re_ip_address:</span><br><span class="line">result = <span class="string">f"<span class="subst">&#123;adress&#125;</span>:<span class="subst">&#123;port&#125;</span>"</span></span><br><span class="line"><span class="keyword">yield</span> result.strip()</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>至此，代理网站的代码已经全部编写完成，接下来我们需要了解的是然后调用此类，直接调用吗？ 显然不是，我们还需要编写一个运行调用的函数，代码如下：</p>
                <figure class="highlight routeros">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">def <span class="builtin-name">run</span>(self):</span><br><span class="line"><span class="comment"># print(self.__CrawlFunc__)</span></span><br><span class="line">proxies = []</span><br><span class="line">callback = self.CrawlFunc_List</span><br><span class="line"><span class="keyword">for</span> i <span class="keyword">in</span> callback:</span><br><span class="line"><span class="builtin-name">print</span>(<span class="string">'Callback'</span>, i)</span><br><span class="line"><span class="keyword">for</span><span class="built_in"> proxy </span><span class="keyword">in</span> eval(<span class="string">"self.&#123;&#125;()"</span>.format(i)):</span><br><span class="line"><span class="builtin-name">print</span>(<span class="string">'Getting'</span>, proxy, <span class="string">'from'</span>, i)</span><br><span class="line">proxies.append(proxy)</span><br><span class="line">return proxies</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>以上代理的代码中，我并没有抓取每个网站全部页数，如果有需要可以自行调节。 这里我们可以直接写一个调用代码，来测试代码是否正常，写在：<strong>Spider.py</strong> 也就是代理的代码中，代码如下：</p>
                <figure class="highlight dockerfile">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">if __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">Tester = FreeProxyGetter()</span><br><span class="line">Tester.<span class="keyword">run</span><span class="bash">()</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>运行结果如下： 结果较多，省略大部分结果。</p>
                <figure class="highlight angelscript">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">Getting <span class="number">14.115</span><span class="number">.70</span><span class="number">.177</span>:<span class="number">4216</span> <span class="keyword">from</span> crawl_xicidaili</span><br><span class="line">Getting <span class="number">116.22</span><span class="number">.48</span><span class="number">.220</span>:<span class="number">4216</span> <span class="keyword">from</span> crawl_xicidaili</span><br><span class="line">Getting <span class="number">223.241</span><span class="number">.3</span><span class="number">.120</span>:<span class="number">4216</span> <span class="keyword">from</span> crawl_xicidaili</span><br><span class="line">......</span><br><span class="line">Getting <span class="number">60.188</span><span class="number">.1</span><span class="number">.27</span>:<span class="number">8232</span> <span class="keyword">from</span> crawl_data5u</span><br><span class="line">Callback crawl_kxdaili</span><br><span class="line">Getting <span class="number">117.71</span><span class="number">.165</span><span class="number">.208</span>:<span class="number">3000</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">223.99</span><span class="number">.197</span><span class="number">.253</span>:<span class="number">63000</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">60.255</span><span class="number">.186</span><span class="number">.169</span>:<span class="number">8888</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">180.168</span><span class="number">.13</span><span class="number">.26</span>:<span class="number">8000</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">125.124</span><span class="number">.51</span><span class="number">.226</span>:<span class="number">808</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">47.99</span><span class="number">.145</span><span class="number">.67</span>:<span class="number">808</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">222.186</span><span class="number">.55</span><span class="number">.41</span>:<span class="number">8080</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">36.6</span><span class="number">.224</span><span class="number">.30</span>:<span class="number">3000</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">110.43</span><span class="number">.42</span><span class="number">.200</span>:<span class="number">8081</span> <span class="keyword">from</span> crawl_kxdaili</span><br><span class="line">Getting <span class="number">39.137</span><span class="number">.69</span><span class="number">.8</span>:<span class="number">80</span> <span class="keyword">from</span> crawl_kxdaili</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>ok，至此代理成功抓取，接下来就是我们的异步检测了。</p>
                <h4 id="4-2-3-异步检测"><a href="#4-2-3-异步检测" class="headerlink" title="4.2.3 异步检测"></a>4.2.3 异步检测</h4>
                <p>这里，我们需要编写一个 <strong>error.py</strong> 来自定义包：</p>
                <figure class="highlight ruby">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ResourceDepletionError</span>(<span class="title">Exception</span>):</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">Exception.__init_<span class="number">_</span>(<span class="keyword">self</span>)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__str__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line"><span class="keyword">return</span> repr(<span class="string">'The proxy source is exhausted'</span>)</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">PoolEmptyError</span>(<span class="title">Exception</span>):</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">Exception.__init_<span class="number">_</span>(<span class="keyword">self</span>)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__str__</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line"><span class="keyword">return</span> repr(<span class="string">'The proxy pool is empty'</span>)</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>异步检测代码，这里我就不做任何数据存储，有需求的可以自行添加，下一篇将添加一个存入 redis 数据库的，敬请关注！</p>
                <figure class="highlight python">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="comment"># files:schedule.py</span></span><br><span class="line"><span class="keyword">import</span> asyncio</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">from</span> multiprocessing.context <span class="keyword">import</span> Process</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> aiohttp</span><br><span class="line"><span class="keyword">from</span> db <span class="keyword">import</span> RedisClient</span><br><span class="line"><span class="keyword">from</span> Spider <span class="keyword">import</span> FreeProxyGetter</span><br><span class="line"><span class="keyword">from</span> error <span class="keyword">import</span> ResourceDepletionError</span><br><span class="line"><span class="keyword">from</span> aiohttp <span class="keyword">import</span> ServerDisconnectedError, ClientResponseError, ClientConnectorError</span><br><span class="line"><span class="keyword">from</span> socks <span class="keyword">import</span> ProxyConnectionError</span><br><span class="line"></span><br><span class="line">get_proxy_timeout = <span class="number">9</span></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">ValidityTester</span><span class="params">(object)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">代理的有效性测试</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span></span><br><span class="line">self.raw_proxies = <span class="literal">None</span></span><br><span class="line">self.usable_proxies = [] <span class="comment"># 可用代理</span></span><br><span class="line">self.test_api = <span class="string">'http://www.baidu.com'</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">set_raw_proxies</span><span class="params">(self, proxies)</span>:</span></span><br><span class="line">self.raw_proxies = proxies <span class="comment"># 临时存储一些代理</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">async</span> <span class="function"><span class="keyword">def</span> <span class="title">test_single_proxy</span><span class="params">(self, proxy)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">python3.5 之后出现的新特性</span></span><br><span class="line"><span class="string">text one proxy, if valid, put them to usable_proxies.</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line"><span class="keyword">async</span> <span class="keyword">with</span> aiohttp.ClientSession() <span class="keyword">as</span> session:</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line"><span class="keyword">if</span> isinstance(proxy, bytes):</span><br><span class="line">proxy = proxy.decode(<span class="string">'utf-8'</span>)</span><br><span class="line">real_proxy = <span class="string">'http://'</span> + proxy</span><br><span class="line">print(<span class="string">'Testing'</span>, proxy)</span><br><span class="line"><span class="keyword">async</span> <span class="keyword">with</span> session.get(self.test_api, proxy=real_proxy, timeout=get_proxy_timeout) <span class="keyword">as</span> response:</span><br><span class="line"><span class="keyword">if</span> response.status == <span class="number">200</span>:</span><br><span class="line">print(<span class="string">'Valid proxy'</span>, proxy)</span><br><span class="line"><span class="keyword">except</span> (ProxyConnectionError, TimeoutError, ValueError):</span><br><span class="line">print(<span class="string">'Invalid proxy'</span>, proxy)</span><br><span class="line"><span class="keyword">except</span> (ServerDisconnectedError, ClientResponseError, ClientConnectorError) <span class="keyword">as</span> s:</span><br><span class="line">print(s)</span><br><span class="line"><span class="keyword">pass</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">test</span><span class="params">(self)</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">aio test all proxies.</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line">print(<span class="string">'ValidityTester is working'</span>)</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">loop = asyncio.get_event_loop()</span><br><span class="line">tasks = [self.test_single_proxy(proxy) <span class="keyword">for</span> proxy <span class="keyword">in</span> self.raw_proxies]</span><br><span class="line">loop.run_until_complete(asyncio.wait(tasks))</span><br><span class="line"><span class="keyword">except</span> ValueError:</span><br><span class="line">print(<span class="string">'Async Error'</span>)</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Schedule</span><span class="params">(object)</span>:</span></span><br><span class="line"><span class="meta">@staticmethod</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">valid_proxy</span><span class="params">()</span>:</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="string">Get half of proxies which in Spider</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line">tester = ValidityTester()</span><br><span class="line">free_proxy_getter = FreeProxyGetter()</span><br><span class="line"><span class="comment"># 获取 Spider 数据库数据</span></span><br><span class="line"><span class="comment"># 调用测试类</span></span><br><span class="line">tester.set_raw_proxies(free_proxy_getter.run())</span><br><span class="line"><span class="comment"># 开始测试</span></span><br><span class="line">tester.test()</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">run</span><span class="params">(self)</span>:</span></span><br><span class="line">print(<span class="string">'Ip processing running'</span>)</span><br><span class="line">valid_process = Process(target=Schedule.valid_proxy) <span class="comment"># 获取代理并筛选</span></span><br><span class="line">valid_process.start()</span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">schedule = Schedule().run()</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>对于上面的部分代理代码，我为了输出结果更加直观，把 <strong>print()</strong> 输出的，包括 <strong>utils.py</strong> 的 <strong>print()</strong> 全部注释了， 运行结果（省略部分输出）</p>
                <figure class="highlight subunit">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="keyword">Testing </span>182.46.252.84:9999</span><br><span class="line"><span class="keyword">Testing </span>183.63.188.250:8808</span><br><span class="line"><span class="keyword">Testing </span>39.137.69.10:8080</span><br><span class="line"><span class="keyword">Testing </span>119.190.149.226:8060</span><br><span class="line"><span class="keyword">Testing </span>219.159.38.201:56210</span><br><span class="line"><span class="keyword">Testing </span>58.246.143.32:8118</span><br><span class="line"><span class="keyword">Testing </span>114.99.117.13:3000</span><br><span class="line"><span class="keyword">Testing </span>114.229.229.248:8118</span><br><span class="line"><span class="keyword">Testing </span>119.57.108.53:53281</span><br><span class="line"><span class="keyword">Testing </span>223.247.94.164:4216</span><br><span class="line"><span class="keyword">Testing </span>219.159.38.208:56210</span><br><span class="line"><span class="keyword">Testing </span>59.44.78.30:42335</span><br><span class="line"><span class="keyword">Testing </span>112.74.87.172:3128</span><br><span class="line"><span class="keyword">Testing </span>182.92.235.109:3128</span><br><span class="line"><span class="keyword">Testing </span>58.62.115.3:4216</span><br><span class="line"><span class="keyword">Testing </span>183.162.171.37:4216</span><br><span class="line"><span class="keyword">Testing </span>223.241.5.46:4216</span><br><span class="line"><span class="keyword">Testing </span>223.243.4.231:4216</span><br><span class="line"><span class="keyword">Testing </span>223.241.5.228:4216</span><br><span class="line"><span class="keyword">Testing </span>183.162.171.56:4216</span><br><span class="line"><span class="keyword">Testing </span>223.241.4.146:4216</span><br><span class="line"><span class="keyword">Testing </span>223.241.4.200:4216</span><br><span class="line"><span class="keyword">Testing </span>223.241.4.182:4216</span><br><span class="line"><span class="keyword">Testing </span>118.187.50.114:8080</span><br><span class="line"><span class="keyword">Testing </span>223.214.176.170:3000</span><br><span class="line"><span class="keyword">Testing </span>125.126.123.208:60004</span><br><span class="line"><span class="keyword">Testing </span>163.125.248.39:8088</span><br><span class="line"><span class="keyword">Testing </span>119.57.108.89:53281</span><br><span class="line"><span class="keyword">Testing </span>222.249.238.138:8080</span><br><span class="line"><span class="keyword">Testing </span>125.126.97.77:60004</span><br><span class="line"><span class="keyword">Testing </span>222.182.54.210:8118</span><br><span class="line">Valid proxy 58.220.95.90:9401</span><br><span class="line">Valid proxy 60.191.45.92:8080</span><br><span class="line">Valid proxy 222.186.55.41:8080</span><br><span class="line">Valid proxy 60.205.132.71:80</span><br><span class="line">Valid proxy 124.90.49.146:8888</span><br><span class="line"></span><br><span class="line">Valid proxy 218.75.109.86:3128</span><br><span class="line">Valid proxy 27.188.64.70:8060</span><br><span class="line"></span><br><span class="line">Valid proxy 27.188.62.3:8060</span><br><span class="line">Valid proxy 221.180.170.104:8080</span><br><span class="line">Valid proxy 39.137.69.9:8080</span><br><span class="line"></span><br><span class="line">Valid proxy 39.137.69.6:80</span><br><span class="line">Valid proxy 39.137.69.10:8080</span><br><span class="line">Valid proxy 114.229.6.215:8118</span><br><span class="line">Valid proxy 221.180.170.104:8080</span><br><span class="line">Valid proxy 218.89.14.142:8060</span><br><span class="line">Valid proxy 116.196.85.150:3128</span><br><span class="line"></span><br><span class="line">Valid proxy 122.224.65.197:3128</span><br><span class="line">Valid proxy 112.253.11.103:8000</span><br><span class="line">Valid proxy 112.253.11.103:8000</span><br><span class="line">Valid proxy 112.253.11.113:8000</span><br><span class="line">Valid proxy 124.90.52.200:8888</span><br><span class="line"></span><br><span class="line">Cannot connect to host 113.103.226.99:4216 ssl:default [Connect call failed ('113.103.226.99', 4216)]</span><br><span class="line">Cannot connect to host 36.249.109.59:8221 ssl:default [Connect call failed ('36.249.109.59', 8221)]</span><br><span class="line">Cannot connect to host 125.126.121.66:60004 ssl:default [Connect call failed ('125.126.121.66', 60004)]</span><br><span class="line">Cannot connect to host 27.42.168.46:48919 ssl:default [Connect call failed ('27.42.168.46', 48919)]</span><br><span class="line">Cannot connect to host 163.125.113.220:8118 ssl:default [Connect call failed ('163.125.113.220', 8118)]</span><br><span class="line">Cannot connect to host 114.55.63.34:808 ssl:default [Connect call failed ('114.55.63.34', 808)]</span><br><span class="line">Cannot connect to host 119.133.207.10:4216 ssl:default [Connect call failed ('119.133.207.10', 4216)]</span><br><span class="line">Cannot connect to host 58.62.115.3:4216 ssl:default [Connect call failed ('58.62.115.3', 4216)]</span><br><span class="line">Cannot connect to host 117.71.164.232:3000 ssl:default [Connect call failed ('117.71.164.232', 3000)]</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>到此，多站抓取异步检测，就已经完全实现，也是实现了模块化编程。我也 顺便把项目结构分享一下：</p>
                <figure class="highlight css">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="selector-tag">C</span><span class="selector-pseudo">:.</span></span><br><span class="line">│ <span class="selector-tag">error</span><span class="selector-class">.py</span></span><br><span class="line">│ <span class="selector-tag">info</span><span class="selector-class">.txt</span></span><br><span class="line">│ <span class="selector-tag">schedule</span><span class="selector-class">.py</span></span><br><span class="line">│ <span class="selector-tag">Spider</span><span class="selector-class">.py</span></span><br><span class="line">│ <span class="selector-tag">the_test_code_crawler</span><span class="selector-class">.py</span></span><br><span class="line">│ <span class="selector-tag">utils</span><span class="selector-class">.py</span></span><br><span class="line">│</span><br><span class="line">└─__<span class="selector-tag">pycache__</span></span><br><span class="line"><span class="selector-tag">db</span><span class="selector-class">.cpython-37</span><span class="selector-class">.pyc</span></span><br><span class="line"><span class="selector-tag">error</span><span class="selector-class">.cpython-37</span><span class="selector-class">.pyc</span></span><br><span class="line"><span class="selector-tag">Spider</span><span class="selector-class">.cpython-37</span><span class="selector-class">.pyc</span></span><br><span class="line"><span class="selector-tag">utils</span><span class="selector-class">.cpython-37</span><span class="selector-class">.pyc</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>本篇我想你主要学习到的目标也就是开头所说的，这里就不赘述了，下一篇我将带大家把抓取到可用的代理进行存储到 redis 数据库中，尽请关注！ 源代码文件获取：公众号后台回放：ProxyPool-1 也欢迎加入我的交流群，一起交流！</p>
              </div>
              <div class="reward-container">
                <div></div>
                <button onclick="var qr = document.getElementById('qr'); qr.style.display = (qr.style.display === 'none') ? 'block' : 'none';"> 打赏 </button>
                <div id="qr" style="display: none;">
                  <div style="display: inline-block;">
                    <img src="/images/wechatpay.jpg" alt="崔庆才 微信支付">
                    <p>微信支付</p>
                  </div>
                  <div style="display: inline-block;">
                    <img src="/images/alipay.jpg" alt="崔庆才 支付宝">
                    <p>支付宝</p>
                  </div>
                </div>
              </div>
              <footer class="post-footer">
                <div class="post-nav">
                  <div class="post-nav-item">
                    <a href="/9533.html" rel="prev" title="Pyppeteer 如何打包 Docker">
                      <i class="fa fa-chevron-left"></i> Pyppeteer 如何打包 Docker </a>
                  </div>
                  <div class="post-nav-item">
                    <a href="/9544.html" rel="next" title="浅入浅出Hash算法"> 浅入浅出Hash算法 <i class="fa fa-chevron-right"></i>
                    </a>
                  </div>
                </div>
              </footer>
            </article>
          </div>
          <div class="comments" id="gitalk-container"></div>
          <script>
            window.addEventListener('tabs:register', () =>
            {
              let
              {
                activeClass
              } = CONFIG.comments;
              if (CONFIG.comments.storage)
              {
                activeClass = localStorage.getItem('comments_active') || activeClass;
              }
              if (activeClass)
              {
                let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
                if (activeTab)
                {
                  activeTab.click();
                }
              }
            });
            if (CONFIG.comments.storage)
            {
              window.addEventListener('tabs:click', event =>
              {
                if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
                let commentClass = event.target.classList[1];
                localStorage.setItem('comments_active', commentClass);
              });
            }

          </script>
        </div>
        <div class="toggle sidebar-toggle">
          <span class="toggle-line toggle-line-first"></span>
          <span class="toggle-line toggle-line-middle"></span>
          <span class="toggle-line toggle-line-last"></span>
        </div>
        <aside class="sidebar">
          <div class="sidebar-inner">
            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc"> 文章目录 </li>
              <li class="sidebar-nav-overview"> 站点概览 </li>
            </ul>
            <!--noindex-->
            <div class="post-toc-wrap sidebar-panel">
              <div class="post-toc motion-element">
                <ol class="nav">
                  <li class="nav-item nav-level-2"><a class="nav-link" href="#1-看完之后你会得到什么"><span class="nav-number">1.</span> <span class="nav-text">1. 看完之后你会得到什么</span></a></li>
                  <li class="nav-item nav-level-2"><a class="nav-link" href="#2-你需要的准备"><span class="nav-number">2.</span> <span class="nav-text">2. 你需要的准备</span></a></li>
                  <li class="nav-item nav-level-2"><a class="nav-link" href="#3-课前预习知识点"><span class="nav-number">3.</span> <span class="nav-text">3. 课前预习知识点</span></a>
                    <ol class="nav-child">
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#3-1-Python-的元类编程"><span class="nav-number">3.1.</span> <span class="nav-text">3.1 Python 的元类编程</span></a>
                        <ol class="nav-child">
                          <li class="nav-item nav-level-4"><a class="nav-link" href="#3-1-1-type"><span class="nav-number">3.1.1.</span> <span class="nav-text">3.1.1 type()</span></a></li>
                          <li class="nav-item nav-level-4"><a class="nav-link" href="#3-1-2-metaclass"><span class="nav-number">3.1.2.</span> <span class="nav-text">3.1.2 metaclass</span></a>
                            <ol class="nav-child">
                              <li class="nav-item nav-level-6"><a class="nav-link" href="#metaclass-是-Python-面向对象里最难理解，也是最难使用的魔术代码。正常情况下，你不会碰到需要使用metaclass的情况，所以，以下内容看不懂也没关系，因为基本上你不会用到。（然而还是被我遇见了，而且还是看不懂，但经过大佬的指点就只是知道如何使用，但并不了解其中的原理，所以才有了此篇。"><span class="nav-number">3.1.2.0.1.</span> <span class="nav-text">metaclass 是 Python 面向对象里最难理解，也是最难使用的魔术代码。正常情况下，你不会碰到需要使用metaclass的情况，所以，以下内容看不懂也没关系，因为基本上你不会用到。（然而还是被我遇见了，而且还是看不懂，但经过大佬的指点就只是知道如何使用，但并不了解其中的原理，所以才有了此篇。)</span></a></li>
                            </ol>
                          </li>
                        </ol>
                      </li>
                    </ol>
                  </li>
                  <li class="nav-item nav-level-3"><a class="nav-link" href="#3-2-Python-eval-函数"><span class="nav-number">3.2.</span> <span class="nav-text">3.2 Python eval() 函数</span></a></li>
                </ol>
                </li>
                <li class="nav-item nav-level-2"><a class="nav-link" href="#4-抓取免费代理"><span class="nav-number">4.</span> <span class="nav-text">4. 抓取免费代理</span></a>
                  <ol class="nav-child">
                    <li class="nav-item nav-level-3"><a class="nav-link" href="#4-1-目标的代理网站"><span class="nav-number">4.1.</span> <span class="nav-text">4.1 目标的代理网站</span></a></li>
                    <li class="nav-item nav-level-3"><a class="nav-link" href="#4-2-代码编写"><span class="nav-number">4.2.</span> <span class="nav-text">4.2 代码编写</span></a>
                      <ol class="nav-child">
                        <li class="nav-item nav-level-4"><a class="nav-link" href="#4-2-1-请求函数编写"><span class="nav-number">4.2.1.</span> <span class="nav-text">4.2.1 请求函数编写</span></a></li>
                        <li class="nav-item nav-level-4"><a class="nav-link" href="#4-2-2-代理抓取代码"><span class="nav-number">4.2.2.</span> <span class="nav-text">4.2.2 代理抓取代码</span></a>
                          <ol class="nav-child">
                            <li class="nav-item nav-level-5"><a class="nav-link" href="#1-快代理"><span class="nav-number">4.2.2.1.</span> <span class="nav-text">1. 快代理</span></a></li>
                            <li class="nav-item nav-level-5"><a class="nav-link" href="#2-西刺代理"><span class="nav-number">4.2.2.2.</span> <span class="nav-text">2. 西刺代理</span></a></li>
                            <li class="nav-item nav-level-5"><a class="nav-link" href="#3-66代理"><span class="nav-number">4.2.2.3.</span> <span class="nav-text">3. 66代理</span></a></li>
                            <li class="nav-item nav-level-5"><a class="nav-link" href="#4-无忧代理"><span class="nav-number">4.2.2.4.</span> <span class="nav-text">4. 无忧代理</span></a></li>
                            <li class="nav-item nav-level-5"><a class="nav-link" href="#5-开心代理-高匿"><span class="nav-number">4.2.2.5.</span> <span class="nav-text">5. 开心代理-高匿</span></a></li>
                            <li class="nav-item nav-level-5"><a class="nav-link" href="#6-云代理"><span class="nav-number">4.2.2.6.</span> <span class="nav-text">6. 云代理</span></a></li>
                          </ol>
                        </li>
                        <li class="nav-item nav-level-4"><a class="nav-link" href="#4-2-3-异步检测"><span class="nav-number">4.2.3.</span> <span class="nav-text">4.2.3 异步检测</span></a></li>
                      </ol>
                    </li>
                  </ol>
                </li>
                </ol>
              </div>
            </div>
            <!--/noindex-->
            <div class="site-overview-wrap sidebar-panel">
              <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
                <img class="site-author-image" itemprop="image" alt="崔庆才" src="/images/avatar.png">
                <p class="site-author-name" itemprop="name">崔庆才</p>
                <div class="site-description" itemprop="description">崔庆才的个人站点，记录生活的瞬间，分享学习的心得。</div>
              </div>
              <div class="site-state-wrap motion-element">
                <nav class="site-state">
                  <div class="site-state-item site-state-posts">
                    <a href="/archives/">
                      <span class="site-state-item-count">608</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>
                  <div class="site-state-item site-state-categories">
                    <a href="/categories/">
                      <span class="site-state-item-count">24</span>
                      <span class="site-state-item-name">分类</span></a>
                  </div>
                  <div class="site-state-item site-state-tags">
                    <a href="/tags/">
                      <span class="site-state-item-count">156</span>
                      <span class="site-state-item-name">标签</span></a>
                  </div>
                </nav>
              </div>
              <div class="links-of-author motion-element">
                <span class="links-of-author-item">
                  <a href="https://github.com/Germey" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;Germey" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
                </span>
                <span class="links-of-author-item">
                  <a href="mailto:cqc@cuiqingcai.com.com" title="邮件 → mailto:cqc@cuiqingcai.com.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>邮件</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://weibo.com/cuiqingcai" title="微博 → https:&#x2F;&#x2F;weibo.com&#x2F;cuiqingcai" rel="noopener" target="_blank"><i class="fab fa-weibo fa-fw"></i>微博</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/Germey" title="知乎 → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;Germey" rel="noopener" target="_blank"><i class="fa fa-magic fa-fw"></i>知乎</a>
                </span>
              </div>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/bco2a.png" style=" width: 100%;">
              </a>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/0ywun.png" style=" width: 100%;">
              </a>
            </div>
            <div class="sidebar-panel sidebar-panel-tags sidebar-panel-active">
              <h4 class="name"> 标签云 </h4>
              <div class="content">
                <a href="/tags/2048/" style="font-size: 10px;">2048</a> <a href="/tags/API/" style="font-size: 10px;">API</a> <a href="/tags/Bootstrap/" style="font-size: 11.25px;">Bootstrap</a> <a href="/tags/CDN/" style="font-size: 10px;">CDN</a> <a href="/tags/CQC/" style="font-size: 10px;">CQC</a> <a href="/tags/CSS/" style="font-size: 10px;">CSS</a> <a href="/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">CSS 反爬虫</a> <a href="/tags/CV/" style="font-size: 10px;">CV</a> <a href="/tags/Django/" style="font-size: 10px;">Django</a> <a href="/tags/Eclipse/" style="font-size: 11.25px;">Eclipse</a> <a href="/tags/FTP/" style="font-size: 10px;">FTP</a> <a href="/tags/Git/" style="font-size: 10px;">Git</a> <a href="/tags/GitHub/" style="font-size: 13.75px;">GitHub</a> <a href="/tags/HTML5/" style="font-size: 10px;">HTML5</a> <a href="/tags/Hexo/" style="font-size: 10px;">Hexo</a> <a href="/tags/IT/" style="font-size: 10px;">IT</a> <a href="/tags/JSP/" style="font-size: 10px;">JSP</a> <a href="/tags/JavaScript/" style="font-size: 10px;">JavaScript</a> <a href="/tags/K8s/" style="font-size: 10px;">K8s</a> <a href="/tags/LOGO/" style="font-size: 10px;">LOGO</a> <a href="/tags/Linux/" style="font-size: 10px;">Linux</a> <a href="/tags/MIUI/" style="font-size: 10px;">MIUI</a> <a href="/tags/MongoDB/" style="font-size: 10px;">MongoDB</a> <a href="/tags/Mysql/" style="font-size: 10px;">Mysql</a> <a href="/tags/NBA/" style="font-size: 10px;">NBA</a> <a href="/tags/PHP/" style="font-size: 11.25px;">PHP</a> <a href="/tags/PS/" style="font-size: 10px;">PS</a> <a href="/tags/Pathlib/" style="font-size: 10px;">Pathlib</a> <a href="/tags/PhantomJS/" style="font-size: 10px;">PhantomJS</a> <a href="/tags/Python/" style="font-size: 15px;">Python</a> <a href="/tags/Python3/" style="font-size: 12.5px;">Python3</a> <a href="/tags/Pythonic/" style="font-size: 10px;">Pythonic</a> <a href="/tags/QQ/" style="font-size: 10px;">QQ</a> <a href="/tags/Redis/" style="font-size: 10px;">Redis</a> <a href="/tags/SAE/" style="font-size: 10px;">SAE</a> <a href="/tags/SSH/" style="font-size: 10px;">SSH</a> <a href="/tags/SVG/" style="font-size: 10px;">SVG</a> <a href="/tags/Scrapy/" style="font-size: 10px;">Scrapy</a> <a href="/tags/Scrapy-redis/" style="font-size: 10px;">Scrapy-redis</a> <a href="/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">Scrapy分布式</a> <a href="/tags/Selenium/" style="font-size: 10px;">Selenium</a> <a href="/tags/TKE/" style="font-size: 10px;">TKE</a> <a href="/tags/Ubuntu/" style="font-size: 11.25px;">Ubuntu</a> <a href="/tags/VS-Code/" style="font-size: 10px;">VS Code</a> <a href="/tags/Vs-Code/" style="font-size: 10px;">Vs Code</a> <a href="/tags/Vue/" style="font-size: 11.25px;">Vue</a> <a href="/tags/Webpack/" style="font-size: 10px;">Webpack</a> <a href="/tags/Windows/" style="font-size: 10px;">Windows</a> <a href="/tags/Winpcap/" style="font-size: 10px;">Winpcap</a> <a href="/tags/WordPress/" style="font-size: 13.75px;">WordPress</a> <a href="/tags/Youtube/" style="font-size: 11.25px;">Youtube</a> <a href="/tags/android/" style="font-size: 10px;">android</a> <a href="/tags/ansible/" style="font-size: 10px;">ansible</a> <a href="/tags/cocos2d-x/" style="font-size: 10px;">cocos2d-x</a> <a href="/tags/e6/" style="font-size: 10px;">e6</a> <a href="/tags/fitvids/" style="font-size: 10px;">fitvids</a> <a href="/tags/git/" style="font-size: 11.25px;">git</a> <a href="/tags/json/" style="font-size: 10px;">json</a> <a href="/tags/js%E9%80%86%E5%90%91/" style="font-size: 10px;">js逆向</a> <a href="/tags/kubernetes/" style="font-size: 10px;">kubernetes</a> <a href="/tags/log/" style="font-size: 10px;">log</a> <a href="/tags/logging/" style="font-size: 10px;">logging</a> <a href="/tags/matlab/" style="font-size: 11.25px;">matlab</a> <a href="/tags/python/" style="font-size: 20px;">python</a> <a href="/tags/pytube/" style="font-size: 11.25px;">pytube</a> <a href="/tags/pywin32/" style="font-size: 10px;">pywin32</a> <a href="/tags/style/" style="font-size: 10px;">style</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/ubuntu/" style="font-size: 10px;">ubuntu</a> <a href="/tags/uwsgi/" style="font-size: 10px;">uwsgi</a> <a href="/tags/vsftpd/" style="font-size: 10px;">vsftpd</a> <a href="/tags/wamp/" style="font-size: 10px;">wamp</a> <a href="/tags/wineQQ/" style="font-size: 10px;">wineQQ</a> <a href="/tags/%E4%B8%83%E7%89%9B/" style="font-size: 11.25px;">七牛</a> <a href="/tags/%E4%B8%8A%E6%B5%B7/" style="font-size: 10px;">上海</a> <a href="/tags/%E4%B8%AA%E4%BA%BA%E7%BD%91%E7%AB%99/" style="font-size: 10px;">个人网站</a> <a href="/tags/%E4%B8%BB%E9%A2%98/" style="font-size: 10px;">主题</a> <a href="/tags/%E4%BA%91%E4%BA%A7%E5%93%81/" style="font-size: 10px;">云产品</a> <a href="/tags/%E4%BA%91%E5%AD%98%E5%82%A8/" style="font-size: 10px;">云存储</a> <a href="/tags/%E4%BA%AC%E4%B8%9C%E4%BA%91/" style="font-size: 10px;">京东云</a> <a href="/tags/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/" style="font-size: 12.5px;">人工智能</a> <a href="/tags/%E4%BB%A3%E7%90%86/" style="font-size: 10px;">代理</a> <a href="/tags/%E4%BB%A3%E7%A0%81/" style="font-size: 10px;">代码</a> <a href="/tags/%E4%BB%A3%E7%A0%81%E5%88%86%E4%BA%AB%E5%9B%BE/" style="font-size: 10px;">代码分享图</a> <a href="/tags/%E4%BC%98%E5%8C%96/" style="font-size: 10px;">优化</a> <a href="/tags/%E4%BD%8D%E8%BF%90%E7%AE%97/" style="font-size: 10px;">位运算</a> <a href="/tags/%E5%85%AC%E4%BC%97%E5%8F%B7/" style="font-size: 10px;">公众号</a> <a href="/tags/%E5%88%86%E4%BA%AB/" style="font-size: 10px;">分享</a> <a href="/tags/%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">分布式</a> <a href="/tags/%E5%88%9B%E4%B8%9A/" style="font-size: 10px;">创业</a> <a href="/tags/%E5%89%8D%E7%AB%AF/" style="font-size: 12.5px;">前端</a> <a href="/tags/%E5%8D%9A%E5%AE%A2/" style="font-size: 10px;">博客</a> <a href="/tags/%E5%8E%9F%E7%94%9FAPP/" style="font-size: 10px;">原生APP</a> <a href="/tags/%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 12.5px;">反爬虫</a> <a href="/tags/%E5%91%BD%E4%BB%A4/" style="font-size: 10px;">命令</a> <a href="/tags/%E5%93%8D%E5%BA%94%E5%BC%8F%E5%B8%83%E5%B1%80/" style="font-size: 10px;">响应式布局</a> <a href="/tags/%E5%9E%83%E5%9C%BE%E9%82%AE%E4%BB%B6/" style="font-size: 10px;">垃圾邮件</a> <a href="/tags/%E5%9F%9F%E5%90%8D%E7%BB%91%E5%AE%9A/" style="font-size: 10px;">域名绑定</a> <a href="/tags/%E5%A4%8D%E7%9B%98/" style="font-size: 10px;">复盘</a> <a href="/tags/%E5%A4%A7%E4%BC%97%E7%82%B9%E8%AF%84/" style="font-size: 10px;">大众点评</a> <a href="/tags/%E5%AD%97%E4%BD%93%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">字体反爬虫</a> <a href="/tags/%E5%AD%97%E7%AC%A6%E9%97%AE%E9%A2%98/" style="font-size: 10px;">字符问题</a> <a href="/tags/%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95/" style="font-size: 10px;">学习方法</a> <a href="/tags/%E5%AE%89%E5%8D%93/" style="font-size: 10px;">安卓</a> <a href="/tags/%E5%AE%9E%E7%94%A8/" style="font-size: 10px;">实用</a> <a href="/tags/%E5%B0%81%E9%9D%A2/" style="font-size: 10px;">封面</a> <a href="/tags/%E5%B4%94%E5%BA%86%E6%89%8D/" style="font-size: 18.75px;">崔庆才</a> <a href="/tags/%E5%B7%A5%E5%85%B7/" style="font-size: 12.5px;">工具</a> <a href="/tags/%E5%BC%80%E5%8F%91%E5%B7%A5%E5%85%B7/" style="font-size: 10px;">开发工具</a> <a href="/tags/%E5%BE%AE%E8%BD%AF/" style="font-size: 10px;">微软</a> <a href="/tags/%E6%80%9D%E8%80%83/" style="font-size: 10px;">思考</a> <a href="/tags/%E6%89%8B%E6%9C%BA%E8%AE%BF%E9%97%AE/" style="font-size: 10px;">手机访问</a> <a href="/tags/%E6%95%99%E7%A8%8B/" style="font-size: 10px;">教程</a> <a href="/tags/%E6%95%99%E8%82%B2/" style="font-size: 10px;">教育</a> <a href="/tags/%E6%96%B0%E4%B9%A6/" style="font-size: 12.5px;">新书</a> <a href="/tags/%E6%96%B9%E6%B3%95%E8%AE%BA/" style="font-size: 10px;">方法论</a> <a href="/tags/%E6%97%85%E6%B8%B8/" style="font-size: 10px;">旅游</a> <a href="/tags/%E6%97%A5%E5%BF%97/" style="font-size: 10px;">日志</a> <a href="/tags/%E6%9A%97%E6%97%B6%E9%97%B4/" style="font-size: 10px;">暗时间</a> <a href="/tags/%E6%9D%9C%E5%85%B0%E7%89%B9/" style="font-size: 11.25px;">杜兰特</a> <a href="/tags/%E6%A1%8C%E9%9D%A2/" style="font-size: 10px;">桌面</a> <a href="/tags/%E6%AD%8C%E5%8D%95/" style="font-size: 10px;">歌单</a> <a href="/tags/%E6%B1%9F%E5%8D%97/" style="font-size: 10px;">江南</a> <a href="/tags/%E6%B8%B8%E6%88%8F/" style="font-size: 10px;">游戏</a> <a href="/tags/%E7%84%A6%E8%99%91/" style="font-size: 10px;">焦虑</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 16.25px;">爬虫</a> <a href="/tags/%E7%88%AC%E8%99%AB%E4%B9%A6%E7%B1%8D/" style="font-size: 11.25px;">爬虫书籍</a> <a href="/tags/%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F/" style="font-size: 10px;">环境变量</a> <a href="/tags/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/" style="font-size: 10px;">生活笔记</a> <a href="/tags/%E7%99%BB%E5%BD%95/" style="font-size: 10px;">登录</a> <a href="/tags/%E7%9F%A5%E4%B9%8E/" style="font-size: 10px;">知乎</a> <a href="/tags/%E7%9F%AD%E4%BF%A1/" style="font-size: 10px;">短信</a> <a href="/tags/%E7%9F%AD%E4%BF%A1%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">短信验证码</a> <a href="/tags/%E7%AC%94%E8%AE%B0%E8%BD%AF%E4%BB%B6/" style="font-size: 10px;">笔记软件</a> <a href="/tags/%E7%AF%AE%E7%BD%91/" style="font-size: 10px;">篮网</a> <a href="/tags/%E7%BA%B8%E5%BC%A0/" style="font-size: 10px;">纸张</a> <a href="/tags/%E7%BB%84%E4%BB%B6/" style="font-size: 10px;">组件</a> <a href="/tags/%E7%BD%91%E7%AB%99/" style="font-size: 10px;">网站</a> <a href="/tags/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/" style="font-size: 11.25px;">网络爬虫</a> <a href="/tags/%E7%BE%8E%E5%AD%A6/" style="font-size: 10px;">美学</a> <a href="/tags/%E8%82%89%E5%A4%B9%E9%A6%8D/" style="font-size: 10px;">肉夹馍</a> <a href="/tags/%E8%85%BE%E8%AE%AF%E4%BA%91/" style="font-size: 10px;">腾讯云</a> <a href="/tags/%E8%87%AA%E5%BE%8B/" style="font-size: 10px;">自律</a> <a href="/tags/%E8%A5%BF%E5%B0%91%E7%88%B7/" style="font-size: 10px;">西少爷</a> <a href="/tags/%E8%A7%86%E9%A2%91/" style="font-size: 10px;">视频</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">谷歌验证码</a> <a href="/tags/%E8%BF%90%E8%90%A5/" style="font-size: 10px;">运营</a> <a href="/tags/%E8%BF%9C%E7%A8%8B/" style="font-size: 10px;">远程</a> <a href="/tags/%E9%80%86%E5%90%91/" style="font-size: 10px;">逆向</a> <a href="/tags/%E9%85%8D%E7%BD%AE/" style="font-size: 10px;">配置</a> <a href="/tags/%E9%87%8D%E8%A3%85/" style="font-size: 10px;">重装</a> <a href="/tags/%E9%98%BF%E6%9D%9C/" style="font-size: 10px;">阿杜</a> <a href="/tags/%E9%9D%99%E8%A7%85/" style="font-size: 17.5px;">静觅</a> <a href="/tags/%E9%A2%A0%E8%A6%86/" style="font-size: 10px;">颠覆</a> <a href="/tags/%E9%A3%9E%E4%BF%A1/" style="font-size: 10px;">飞信</a> <a href="/tags/%E9%B8%BF%E8%92%99/" style="font-size: 10px;">鸿蒙</a>
              </div>
              <script>
                const tagsColors = ['#00a67c', '#5cb85c', '#d9534f', '#567e95', '#b37333', '#f4843d', '#15a287']
                const tagsElements = document.querySelectorAll('.sidebar-panel-tags .content a')
                tagsElements.forEach((item) =>
                {
                  item.style.backgroundColor = tagsColors[Math.floor(Math.random() * tagsColors.length)]
                })

              </script>
            </div>
            <div class="sidebar-panel sidebar-panel-categories sidebar-panel-active">
              <h4 class="name"> 分类 </h4>
              <div class="content">
                <ul class="category-list">
                  <li class="category-list-item"><a class="category-list-link" href="/categories/C-C/">C/C++</a><span class="category-list-count">23</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/HTML/">HTML</a><span class="category-list-count">14</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Java/">Java</a><span class="category-list-count">5</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/JavaScript/">JavaScript</a><span class="category-list-count">26</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Linux/">Linux</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Markdown/">Markdown</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Net/">Net</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Other/">Other</a><span class="category-list-count">39</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/PHP/">PHP</a><span class="category-list-count">27</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Paper/">Paper</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Python/">Python</a><span class="category-list-count">261</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/TypeScript/">TypeScript</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/">个人展示</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/">个人日记</a><span class="category-list-count">9</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/">个人记录</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/">个人随笔</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">安装配置</a><span class="category-list-count">59</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/">技术杂谈</a><span class="category-list-count">88</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/">未分类</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/">生活笔记</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/">福利专区</a><span class="category-list-count">6</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/">职位推荐</a><span class="category-list-count">2</span></li>
                </ul>
              </div>
            </div>
            <div class="sidebar-panel sidebar-panel-friends sidebar-panel-active">
              <h4 class="name"> 友情链接 </h4>
              <ul class="friends">
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/j2dub.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.findhao.net/" target="_blank" rel="noopener">FindHao</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ou6mm.jpg">
                  </span>
                  <span class="link">
                    <a href="https://diygod.me/" target="_blank" rel="noopener">DIYgod</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/6apxu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.51dev.com/" target="_blank" rel="noopener">IT技术社区</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.jankl.com/img/titleshu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.jankl.com/" target="_blank" rel="noopener">liberalist</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/bqlbs.png">
                  </span>
                  <span class="link">
                    <a href="http://www.urselect.com/" target="_blank" rel="noopener">优社电商</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8s88c.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yuanrenxue.com/" target="_blank" rel="noopener">猿人学</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2wgg5.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yunlifang.cn/" target="_blank" rel="noopener">云立方</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/shwr6.png">
                  </span>
                  <span class="link">
                    <a href="http://lanbing510.info/" target="_blank" rel="noopener">冰蓝</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/blvoh.jpg">
                  </span>
                  <span class="link">
                    <a href="https://lengyue.me/" target="_blank" rel="noopener">冷月</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="http://qianxunclub.com/favicon.png">
                  </span>
                  <span class="link">
                    <a href="http://qianxunclub.com/" target="_blank" rel="noopener">千寻啊千寻</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/0044u.jpg">
                  </span>
                  <span class="link">
                    <a href="http://kodcloud.com/" target="_blank" rel="noopener">可道云</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ygnpn.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.kunkundashen.cn/" target="_blank" rel="noopener">坤坤大神</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/22uv1.png">
                  </span>
                  <span class="link">
                    <a href="http://www.cenchong.com/" target="_blank" rel="noopener">岑冲博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ev9kl.png">
                  </span>
                  <span class="link">
                    <a href="http://www.zxiaoji.com/" target="_blank" rel="noopener">张小鸡</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.503error.com/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.503error.com/" target="_blank" rel="noopener">张志明个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/x714o.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.hubwiz.com/" target="_blank" rel="noopener">汇智网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/129d8.png">
                  </span>
                  <span class="link">
                    <a href="https://www.bysocket.com/" target="_blank" rel="noopener">泥瓦匠BYSocket</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.xiongge.club/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.xiongge.club/" target="_blank" rel="noopener">熊哥club</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/3w4fe.png">
                  </span>
                  <span class="link">
                    <a href="https://zerlong.com/" target="_blank" rel="noopener">知语</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/44hxf.png">
                  </span>
                  <span class="link">
                    <a href="http://redstonewill.com/" target="_blank" rel="noopener">红色石头</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8g1fk.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.laodong.me/" target="_blank" rel="noopener">老董博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/wkaus.jpg">
                  </span>
                  <span class="link">
                    <a href="https://zhaoshuai.me/" target="_blank" rel="noopener">碎念</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/pgo0r.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.chenwenguan.com/" target="_blank" rel="noopener">陈文管的博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/kk82a.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.lxlinux.net/" target="_blank" rel="noopener">良许Linux教程网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/lj0t2.jpg">
                  </span>
                  <span class="link">
                    <a href="https://tanqingbo.cn/" target="_blank" rel="noopener">IT码农</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/i8cdr.png">
                  </span>
                  <span class="link">
                    <a href="https://junyiseo.com/" target="_blank" rel="noopener">均益个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/chwv2.png">
                  </span>
                  <span class="link">
                    <a href="https://brucedone.com/" target="_blank" rel="noopener">大鱼的鱼塘</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2y43o.png">
                  </span>
                  <span class="link">
                    <a href="http://bbs.nightteam.cn/" target="_blank" rel="noopener">夜幕爬虫安全论坛</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/zvc3w.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.weishidong.com/" target="_blank" rel="noopener">韦世东的技术专栏</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ebudy.jpg">
                  </span>
                  <span class="link">
                    <a href="https://chuanjiabing.com/" target="_blank" rel="noopener">穿甲兵技术社区</a>
                  </span>
                </li>
              </ul>
            </div>
          </div>
        </aside>
        <div id="sidebar-dimmer"></div>
      </div>
    </main>
    <footer class="footer">
      <div class="footer-inner">
        <div class="copyright"> &copy; <span itemprop="copyrightYear">2021</span>
          <span class="with-love">
            <i class="fa fa-heart"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">崔庆才丨静觅</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-chart-area"></i>
          </span>
          <span title="站点总字数">2.6m</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>
          <span title="站点阅读时长">39:54</span>
        </div>
        <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动 </div>
        <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">京ICP备18015597号-1 </a>
        </div>
        <script>
          (function ()
          {
            function leancloudSelector(url)
            {
              url = encodeURI(url);
              return document.getElementById(url).querySelector('.leancloud-visitors-count');
            }

            function addCount(Counter)
            {
              var visitors = document.querySelector('.leancloud_visitors');
              var url = decodeURI(visitors.id);
              var title = visitors.dataset.flagTitle;
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                if (results.length > 0)
                {
                  var counter = results[0];
                  leancloudSelector(url).innerText = counter.time + 1;
                  Counter('put', '/classes/Counter/' + counter.objectId,
                  {
                    time:
                    {
                      '__op': 'Increment',
                      'amount': 1
                    }
                  }).catch(error =>
                  {
                    console.error('Failed to save visitor count', error);
                  });
                }
                else
                {
                  Counter('post', '/classes/Counter',
                  {
                    title,
                    url,
                    time: 1
                  }).then(response => response.json()).then(() =>
                  {
                    leancloudSelector(url).innerText = 1;
                  }).catch(error =>
                  {
                    console.error('Failed to create', error);
                  });
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }

            function showTime(Counter)
            {
              var visitors = document.querySelectorAll('.leancloud_visitors');
              var entries = [...visitors].map(element =>
              {
                return decodeURI(element.id);
              });
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url:
                {
                  '$in': entries
                }
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                for (let url of entries)
                {
                  let target = results.find(item => item.url === url);
                  leancloudSelector(url).innerText = target ? target.time : 0;
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }
            let
            {
              app_id,
              app_key,
              server_url
            } = {
              "enable": true,
              "app_id": "6X5dRQ0pnPWJgYy8SXOg0uID-gzGzoHsz",
              "app_key": "ziLDVEy73ne5HtFTiGstzHMS",
              "server_url": "https://6x5drq0p.lc-cn-n1-shared.com",
              "security": false
            };

            function fetchData(api_server)
            {
              var Counter = (method, url, data) =>
              {
                return fetch(`${api_server}/1.1${url}`,
                {
                  method,
                  headers:
                  {
                    'X-LC-Id': app_id,
                    'X-LC-Key': app_key,
                    'Content-Type': 'application/json',
                  },
                  body: JSON.stringify(data)
                });
              };
              if (CONFIG.page.isPost)
              {
                if (CONFIG.hostname !== location.hostname) return;
                addCount(Counter);
              }
              else if (document.querySelectorAll('.post-title-link').length >= 1)
              {
                showTime(Counter);
              }
            }
            let api_server = app_id.slice(-9) !== '-MdYXbMMI' ? server_url : `https://${app_id.slice(0, 8).toLowerCase()}.api.lncldglobal.com`;
            if (api_server)
            {
              fetchData(api_server);
            }
            else
            {
              fetch('https://app-router.leancloud.cn/2/route?appId=' + app_id).then(response => response.json()).then((
              {
                api_server
              }) =>
              {
                fetchData('https://' + api_server);
              });
            }
          })();

        </script>
      </div>
      <div class="footer-stat">
        <span id="cnzz_stat_icon_1279355174"></span>
        <script type="text/javascript">
          document.write(unescape("%3Cspan id='cnzz_stat_icon_1279355174'%3E%3C/span%3E%3Cscript src='https://v1.cnzz.com/z_stat.php%3Fid%3D1279355174%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));

        </script>
      </div>
    </footer>
  </div>
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/pangu@4/dist/browser/pangu.min.js"></script>
  <script src="/js/utils.js"></script>
  <script src="/.js"></script>
  <script src="/js/schemes/pisces.js"></script>
  <script src="/.js"></script>
  <script src="/js/next-boot.js"></script>
  <script src="/.js"></script>
  <script>
    (function ()
    {
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x = document.getElementsByTagName("link");
      //Find the last canonical URL
      if (x.length > 0)
      {
        for (i = 0; i < x.length; i++)
        {
          if (x[i].rel.toLowerCase() == 'canonical' && x[i].href)
          {
            canonicalURL = x[i].href;
          }
        }
      }
      //Get protocol
      if (!canonicalURL)
      {
        curProtocol = window.location.protocol.split(':')[0];
      }
      else
      {
        curProtocol = canonicalURL.split(':')[0];
      }
      //Get current URL if the canonical URL does not exist
      if (!canonicalURL) canonicalURL = window.location.href;
      //Assign script content. Replace current URL with the canonical URL
      ! function ()
      {
        var e = /([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,
          r = canonicalURL,
          t = document.referrer;
        if (!e.test(r))
        {
          var n = (String(curProtocol).toLowerCase() === 'https') ? "https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif" : "//api.share.baidu.com/s.gif";
          t ? (n += "?r=" + encodeURIComponent(document.referrer), r && (n += "&l=" + r)) : r && (n += "?l=" + r);
          var i = new Image;
          i.src = n
        }
      }(window);
    })();

  </script>
  <script src="/js/local-search.js"></script>
  <script src="/.js"></script>
  <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.css">
  <script>
    NexT.utils.loadComments(document.querySelector('#gitalk-container'), () =>
    {
      NexT.utils.getScript('//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.js', () =>
      {
        var gitalk = new Gitalk(
        {
          clientID: '4c86ce1d7c4fbb3b277c',
          clientSecret: '4927beb0f90e2c07e66c99d9d2529cf3eb8ac8e4',
          repo: 'Blog',
          owner: 'germey',
          admin: ['germey'],
          id: '2a580b3e8aed5b2f40001a9a90d11ec5',
          language: 'zh-CN',
          distractionFreeMode: true
        });
        gitalk.render('gitalk-container');
      }, window.Gitalk);
    });

  </script>
</body>

</html>
